package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"golang.org/x/text/encoding"
	"bufio"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding/unicode"
	"log"
	"time"
)

var rateLimiter=time.Tick(100*time.Millisecond)
func Fetch(url string)([] byte,error){
	<-rateLimiter
	resp,err:=http.Get(url)
	if err!=nil{
		return nil,err
	}
	defer resp.Body.Close()

	if resp.StatusCode!=http.StatusOK {
		fmt.Println("Error: status code", resp.StatusCode)
		return nil,
			fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}
	bodyReader:=bufio.NewReader(resp.Body)
	e:=determineEncoding(bodyReader)
	utf8Reader:=transform.NewReader(bodyReader,e.NewDecoder())
	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(
	r *bufio.Reader) encoding.Encoding{
	bytes,err:=r.Peek(1024)
	if err!=nil{
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}
	e,_,_:=charset.DetermineEncoding(bytes,"")
	return e
}
